# -*- coding: utf-8 -*-
# ----------------------------
# @Time    : 2021/6/26 上午11:27
# @Author  : changqingai
# @FileName: pyspark_streaming.py
# ----------------------------

from pyspark import SparkContext
from pyspark.streaming import StreamingContext

if __name__ == "__main__":
    sc = SparkContext(master="local[*]", appName="pyspark-demo")
    ssc = StreamingContext(sc, 5)

    lines = ssc.socketTextStream("node1", 9999)

    res = lines.flatMap(lambda line: line.split(" "))\
        .map(lambda word: (word, 1))\
        .reduceByKey(lambda a, b: a + b)
    res.pprint()

    ssc.start()
    ssc.awaitTermination()
